// --- Copyright (c) 2010 Steven Hartgers ---
// Copyright (C) 2010 Steven Hartgers All rights are reserved.
// Author contact: fiinix00/at\gmail.com
// ------------------------------------------------------------------------
// This program is free software: you can redistribute it and/or modify 
// it under the terms of the GNU General Public License as published by 
// the Free Software Foundation, either version 3 of the License, or 
// (at your option) any later version. 
//  
// This program is distributed in the hope that it will be useful, 
// but WITHOUT ANY WARRANTY; without even the implied warranty of 
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
// GNU General Public License for more detail
//  
// You should have received a copy of the GNU General Public License 
// along with this program.  If not, see <http://www.gnu.org/licenses/>.
// ------------------------------------------------------------------------
// COPYRIGHT NOTICE: This copyright notice may NOT be removed, obscured or modified 
// without written consent from the author.
// --- End copyright notice --- 


// inventor name: Hatem Mostafa
// Date: 19/3/2006

using System;
//using System.Drawing;
using System.Xml;
using System.IO;
using System.Text;
using System.Text.RegularExpressions;
using System.Collections;
using System.ComponentModel;
//using System.Windows.Forms;
using System.Data;
using System.Resources;
using System.Threading;
using System.Runtime.InteropServices;
using Microsoft.Win32;
using System.Net;
using System.Net.Sockets;
using LiteLib;


namespace Hack.Net.Crawler
{
	/// <summary>
	/// Summary description for Form1.
	/// </summary>
	public class HackCrawler// : System.Windows.Forms.Form
	{
        public delegate void StatusChanged(object sender, string status);
        public event StatusChanged OnStatusChanged;

        public delegate void Error(object sender, string strHead, string strBody);
        public event Error OnError;

		// unique Uri's queue
		private Queue queueURLS;
		// thread that take the browse editor text to parse it
		private Thread threadParse;
		// binary tree to keep unique Uri's
		private SortTree urlStorage;
		// Performance Counter to measure CPU usage
		private System.Diagnostics.PerformanceCounter cpuCounter; 
		// Performance Counter to measure memory usage
		private System.Diagnostics.PerformanceCounter ramCounter;

        #region Fields
        // number of bytes downloaded
		private int nByteCount;
		public int ByteCount
		{
			get	{	return nByteCount;	}
			set
			{
				nByteCount = value;
                //this.statusBarPanelByteCount.Text = Commas(nByteCount/1024+1)+ " KB";
			}
		}

		// number of errors during the download process
		private int nErrorCount;
		public int ErrorCount
		{
			get	{	return nErrorCount;	}
			set
			{
				nErrorCount = value;
                //this.statusBarPanelErrors.Text = Commas(nErrorCount) + " errors";
			}
		}

		// number of Uri's found
		private int nURLCount;
		public int URLCount
		{
			get	{	return nURLCount;	}
			set
			{
				nURLCount = value;
                //this.statusBarPanelURLs.Text = Commas(nURLCount)+ " URL found";
			}
		}

		// available memory
		private float nFreeMemory;
		public float FreeMemory
		{
			get	{	return nFreeMemory;	}
			set
			{
				nFreeMemory = value;
                //this.statusBarPanelMem.Text = nFreeMemory + " Mb Available";
			}
		}

		// CPU usage
		private int nCPUUsage;
		public int CPUUsage
		{
			get	{	return nCPUUsage;	}
			set
			{
				nCPUUsage = value;
                //this.statusBarPanelCPU.Text = "CPU usage " + nCPUUsage +"%";

                //Icon icon = Icon.FromHandle(((Bitmap)imageListPercentage.Images[value/10]).GetHicon());
                //this.statusBarPanelCPU.Icon = icon;
			}
		}

		// download folder
		private string strDownloadfolder;
		public string Downloadfolder
		{
			get	{	return strDownloadfolder;	}
			set
			{
				strDownloadfolder = value;
				strDownloadfolder = strDownloadfolder.TrimEnd('\\');
			}
		}

        public string ParseURL
        {
            get;
            set;
        }
		
		// number of files downloaded
		private int nFileCount;
		public int FileCount
		{
			get	{	return nFileCount;	}
			set
			{
				nFileCount = value;
                //this.statusBarPanelFiles.Text = Commas(nFileCount)+ " file(s) downloaded";
			}
		}
		
		// threads array
		private Thread[] threadsRun;

		// number of running threads
		private int nThreadCount;
		public int ThreadCount
		{
			get	{	return nThreadCount;	}
			set
			{
                //Monitor.Enter(this.listViewThreads);
                //try
                //{
                //    for(int nIndex = 0; nIndex < value; nIndex ++)
                //    {
                //        // check if thread not created or not suspended
                //        if(threadsRun[nIndex] == null || threadsRun[nIndex].ThreadState != ThreadState.Suspended)
                //        {	
                //            // create new thread
                //            threadsRun[nIndex] = new Thread(new ThreadStart(ThreadRunFunction));
                //            // set thread name equal to its index
                //            threadsRun[nIndex].Name = nIndex.ToString();
                //            // start thread working function
                //            threadsRun[nIndex].Start();
                //            // check if thread dosn't added to the view
                //            if(nIndex == this.listViewThreads.Items.Count)
                //            {
                //                // add a new line in the view for the new thread
                //                ListViewItem item = this.listViewThreads.Items.Add((nIndex+1).ToString(), 0);
                //                string[] subItems = { "", "", "", "0", "0%" };
                //                item.SubItems.AddRange(subItems);
                //            }
                //        }
                //        // check if the thread is suspended
                //        else	if(threadsRun[nIndex].ThreadState == ThreadState.Suspended)
                //        {
                //            // get thread item from the list
                //            ListViewItem item = this.listViewThreads.Items[nIndex];
                //            item.ImageIndex = 1;
                //            item.SubItems[2].Text = "Resume";
                //            // resume the thread
                //            threadsRun[nIndex].Resume();
                //        }
                //    }
                //    // change thread value
                //    nThreadCount = value;
                //}
                //catch(Exception)
                //{
                //}
                //Monitor.Exit(this.listViewThreads);
			}
		}

		// MIME types string
		private string strMIMETypes = GetMIMETypes();
		public string MIMETypes
		{
			get	{	return strMIMETypes;	}
			set	{	strMIMETypes = value;	}
		}

		// encoding text that includes all settings types in one string
		private Encoding encoding = GetTextEncoding();
		public Encoding TextEncoding
		{
			get	{	return encoding;	}
			set	{	encoding = value;	}
		}

		// timeout of sockets send and receive
		private int nRequestTimeout;
		public int RequestTimeout
		{
			get	{	return nRequestTimeout;	}
			set	{	nRequestTimeout = value;	}
		}

		// the time that each thread sleeps when the refs queue empty
		private int nSleepFetchTime;
		public int SleepFetchTime
		{
			get	{	return nSleepFetchTime;	}
			set	{	nSleepFetchTime = value;	}
		}		
		
		// List of a user defined list of restricted words to enable user to prevent any bad pages 
		private string[] strExcludeWords;
		public string[] ExcludeWords
		{
			get	{	return strExcludeWords;	}
			set	{	strExcludeWords = value;	}
		}

		// List of a user defined list of restricted files extensions to avoid paring non-text data 
		private string[] strExcludeFiles;
		public string[] ExcludeFiles
		{
			get	{	return strExcludeFiles;	}
			set	{	strExcludeFiles = value;	}
		}

		// List of a user defined list of restricted hosts extensions to avoid blocking by these hosts
		private string[] strExcludeHosts;
		public string[] ExcludeHosts
		{
			get	{	return strExcludeHosts;	}
			set	{	strExcludeHosts = value;	}
		}
		
		// the number of requests to keep in the requests view for review requests details
		private int nLastRequestCount;
		public int LastRequestCount
		{
			get	{	return nLastRequestCount;	}
			set	{	nLastRequestCount = value;	}
		}
		
		// the time that each thread sleep after handling any request, 
		// which is very important value to prevent Hosts from blocking the crawler due to heavy load
		private int nSleepConnectTime;
		public int SleepConnectTime
		{
			get	{	return nSleepConnectTime;	}
			set	{	nSleepConnectTime = value;	}
		}

		// represents the depth of navigation in the crawling process
		private int nWebDepth;
		public int WebDepth
		{
			get	{	return nWebDepth;	}
			set	{	nWebDepth = value;	}
		}

		// MIME types are the types that are supported to be downloaded by the crawler 
		// and the crawler includes a default types to be used. 
		private bool bAllMIMETypes;
		public bool AllMIMETypes
		{
			get	{	return bAllMIMETypes;	}
			set	{	bAllMIMETypes = value;	}
		}		

		// to limit crawling process to the same host of the original URL
		private bool bKeepSameServer;
		public bool KeepSameServer
		{
			get	{	return bKeepSameServer;	}
			set	{	bKeepSameServer = value;	}
		}		
		
		// means keep socket connection opened for subsequent requests to avoid reconnect time
		private bool bKeepAlive;
		public bool KeepAlive
		{
			get	{	return bKeepAlive;	}
			set	{	bKeepAlive = value;	}
		}			
		
		// flag to be used to stop all running threads when user request to stop
		bool ThreadsRunning;

        #endregion

        public HackCrawler(Uri url)
		{
            this.ParseURL = url.AbsoluteUri;

			this.urlStorage = new SortTree();
			this.threadsRun = new Thread[200];
			this.queueURLS = new Queue();
			this.cpuCounter = new System.Diagnostics.PerformanceCounter(); 
			this.ramCounter = new System.Diagnostics.PerformanceCounter("Memory", "Available MBytes"); 

			this.cpuCounter.CategoryName = "Processor"; 
			this.cpuCounter.CounterName = "% Processor Time"; 
			this.cpuCounter.InstanceName = "_Total";

            this.WebDepth = 3;//Settings.GetValue("Web depth", 3);
            this.RequestTimeout = 20;//Settings.GetValue("Request timeout", 20);
            this.SleepFetchTime = 2;//Settings.GetValue("Sleep fetch time", 2);
            this.SleepConnectTime = 1;//Settings.GetValue("Sleep connect time", 1);
            this.KeepSameServer = false;//Settings.GetValue("Keep same URL server", false);
            this.AllMIMETypes = true; //Settings.GetValue("Allow all MIME types", true);
            this.KeepAlive = true;//Settings.GetValue("Keep connection alive", true);
            this.ExcludeHosts = new string[] { ".gov", ".org" }; //Settings.GetValue("Exclude Hosts", ".org; .gov;").Replace("*", "").ToLower().Split(';');
            this.ExcludeWords = new string[] { };//Settings.GetValue("Exclude words", "").Split(';');
            this.ExcludeFiles = new string[] { };//Settings.GetValue("Exclude files", "").Replace("*", "").ToLower().Split(';');
            this.LastRequestCount = 20; //Settings.GetValue("View last requests count", 20);
            this.Downloadfolder = System.Environment.SpecialFolder.Personal + "\"Hack.NetCrawler";//Settings.GetValue("Download folder", System.Environment.GetFolderPath(System.Environment.SpecialFolder.Personal));
            this.MIMETypes = GetMIMETypes();
            this.TextEncoding = GetTextEncoding();
		}

		/// <summary>
		/// Clean up any resources being used.
		/// </summary>
		protected void Dispose( bool disposing )
		{
			this.StopParsing();

			if( disposing )
			{
                
                //if (components != null) 
                //{
                //    components.Dispose();
                //}
			}
			//base.Dispose( disposing );

			//System.Environment.Exit(0);
		}



		[DllImport("wininet")]
		public static extern int InternetGetConnectedState(ref int lpdwFlags, int dwReserved);
		[DllImport("wininet")]
		public static extern int InternetAutodial(int dwFlags, int hwndParent);
		int nFirstTimeCheckConnection = 0;
		public string InternetGetConnectedStateString()
		{
			string strState = "";
			try
			{
				int nState = 0;
				// check internet connection state
				if(InternetGetConnectedState(ref nState, 0) == 0)
					return "You are currently not connected to the internet";
				if((nState & 1) == 1)
					strState = "Modem connection";
				else	if((nState & 2) == 2)
					strState = "LAN connection";
				else	if((nState & 4) == 4)
					strState = "Proxy connection";
				else	if((nState & 8) == 8)
					strState = "Modem is busy with a non-Internet connection";
				else	if((nState & 0x10) == 0x10)
					strState = "Remote Access Server is installed";
				else	if((nState & 0x20) == 0x20)
					return "Offline";
				else	if((nState & 0x40) == 0x40)
					return "Internet connection is currently configured";
					
				// get current machine IP
				IPHostEntry he = Dns.Resolve(Dns.GetHostName());
				strState += ",  Machine IP: "+he.AddressList[0].ToString();
			}
			catch
			{
			}
			return strState;
		}
		void ConnectionInfo()
		{
			try
			{
				int nState = 0;
				if(InternetGetConnectedState(ref nState, 0) == 0)
				{
					if(nFirstTimeCheckConnection++ == 0)
						// ask for dial up or DSL connection
						if(InternetAutodial(1, 0) != 0)
							// check internet connection state again
							InternetGetConnectedState(ref nState, 0);
				}
				if((nState & 2) == 2 || (nState & 4) == 4)
					// reset to reask for connection agina
					nFirstTimeCheckConnection = 0;
			}
			catch
			{
			}
			//this.statusBarPanelInfo.Text = InternetGetConnectedStateString();
		}



		private void menuItemSettings_Click(object sender, System.EventArgs e)
		{
			ShowSettings(-1);
		}

		void ShowSettings(int nSelectedIndex)
		{
            //SettingsForm form = new SettingsForm();
            //form.SelectedIndex = nSelectedIndex;
            //if(form.ShowDialog(this) == DialogResult.OK)
            //{
            //    ThreadCount = Settings.GetValue("Threads count", 10);
            //    InitValues();
            //}
		}


		static Encoding GetTextEncoding()
		{
            //Encoding code;
            //if(Settings.GetValue("Use windows default code page", true) == true)
            //    code = Encoding.Default;
            //else
            //{
            //    string strCodePage = Settings.GetValue("Settings code page");
            //    Regex reg = new Regex(@"\([0-9]*\)");
            //    strCodePage = reg.Match(strCodePage).Value;
            //    code = Encoding.GetEncoding(int.Parse(strCodePage.Trim('(', ')')));
            //}
            //return code;
            return Encoding.Default;
		}
		
		// construct MIME types string from settings xml file
		static string GetMIMETypes()
		{
			string str = "";
			// check for settings xml file existence
            //if(File.Exists(Application.StartupPath+"\\Settings.xml"))
            //{
            //    XmlDocument doc = new XmlDocument();
            //    doc.Load(Application.StartupPath+"\\Settings.xml");
            //    XmlNode element = doc.DocumentElement.SelectSingleNode("SettingsForm-listViewFileMatches");
            //    if(element != null)
            //    {
            //        for(int n = 0; n < element.ChildNodes.Count; n++)
            //        {
            //            XmlNode xmlnode  = element.ChildNodes[n];
            //            XmlAttribute attribute  = xmlnode.Attributes["Checked"];
            //            if(attribute == null || attribute.Value.ToLower() != "true")
            //                continue;
            //            string[] items = xmlnode.InnerText.Split('\t');
            //            if(items.Length > 1)
            //            {
            //                str += items[0];
            //                if(items.Length > 2)
            //                    str += '['+items[1]+','+items[2]+']';
            //                str += ';';
            //            }
            //        }
            //    }
            //}
			return str;
		}


		public void StartParsing()
		{			
            //this.buttonGo.Enabled = false;
            //// insert combo text in the combo list
            //if(this.comboBoxWeb.FindStringExact(this.comboBoxWeb.Text) < 0)
            //    this.comboBoxWeb.Items.Insert(0, this.comboBoxWeb.Text);

            if (threadParse == null || threadParse.ThreadState != ThreadState.Suspended)
            {
                this.urlStorage.Clear();
                // start parsing thread
                threadParse = new Thread(new ThreadStart(RunParser));
                threadParse.Start();
            }

            //// update running threads
            ThreadCount = 10;//Settings.GetValue("Threads count", 10);

            //this.toolBarButtonContinue.Enabled = false;
            //this.toolBarButtonPause.Enabled = true;
		}

        public void ContinueParsing()
		{
            if (threadParse == null)
                return;
            if (threadParse.ThreadState == ThreadState.Suspended)
                threadParse.Resume();

            //// update runnning threads
            ThreadCount = 10;//Settings.GetValue("Threads count", 10);

            //this.toolBarButtonContinue.Enabled = false;
            //this.toolBarButtonPause.Enabled = true;
		}

		// pause all working threads
        public void PauseParsing()
		{
            if (threadParse.ThreadState == ThreadState.Running)
                threadParse.Suspend();
            for (int n = 0; n < ThreadCount; n++)
            {
                Thread thread = this.threadsRun[n];
                if (thread.ThreadState == ThreadState.Running)
                    thread.Suspend();
            }
            //this.toolBarButtonContinue.Enabled = true;
            //this.toolBarButtonPause.Enabled = false;
		}
		
		// abort all working threads
        public void StopParsing()
		{
            //this.queueURLS.Clear();
            ThreadsRunning = false;
            Thread.Sleep(500);
            try
            {
                if (threadParse.ThreadState == ThreadState.Suspended)
                    threadParse.Resume();
                threadParse.Abort();
            }
            catch (Exception)
            {
            }
            //Monitor.Enter(this.listViewThreads);
            for (int n = 0; n < ThreadCount; n++)
            {
                try
                {
                    Thread thread = this.threadsRun[n];
                    //ListViewItem itemLog = this.listViewThreads.Items[int.Parse(thread.Name)];
                    //itemLog.SubItems[2].Text = "Stop";
                    //itemLog.BackColor = Color.WhiteSmoke;
                    //itemLog.ImageIndex = 3;
                    if (thread != null && thread.IsAlive)
                    {
                        if (thread.ThreadState == ThreadState.Suspended)
                            thread.Resume();
                        thread.Abort();
                    }
                }
                catch (Exception)
                {
                }
            }
            //Monitor.Exit(this.listViewThreads);
            //this.toolBarButtonContinue.Enabled = true;
            //this.toolBarButtonPause.Enabled = false;
            //this.buttonGo.Enabled = true;

            //this.queueURLS.Clear();
            //this.urlStorage.Clear();
		}

		void ThreadRunFunction()
		{
            MyWebRequest request = null;
            while (ThreadsRunning && int.Parse(Thread.CurrentThread.Name) < this.ThreadCount)
            {
                MyUri uri = DequeueUri();
                if (uri != null)
                {
                    if (SleepConnectTime > 0)
                        Thread.Sleep(SleepConnectTime * 1000);
                    ParseUri(uri, ref request);
                }
                else
                    Thread.Sleep(SleepFetchTime * 1000);
            }

            //Monitor.Enter(this.listViewThreads);
            //try
            //{
            //    ListViewItem item = this.listViewThreads.Items[int.Parse(Thread.CurrentThread.Name)];
            //    if(ThreadsRunning == false)
            //        item.SubItems[2].Text = "Stop";
            //    item.ImageIndex = 0;
            //}
            //catch(Exception)
            //{
            //}
            //Monitor.Exit(this.listViewThreads);
		}

		// push uri to the queue
		bool EnqueueUri(MyUri uri, bool bCheckRepetition)
		{
			// add the uri to the binary tree to check if it is duplicated or not
			if(bCheckRepetition == true && AddURL(ref uri) == false)
				return false;

			Monitor.Enter(queueURLS);
			try
			{
				// add the uri to the queue
				queueURLS.Enqueue(uri);
			}
			catch(Exception)
			{
			}
			Monitor.Exit(queueURLS);

			return true;
		}

		// pop uri from the queue
		MyUri DequeueUri()
		{
			Monitor.Enter(queueURLS);
			MyUri uri = null;
			try
			{
				uri = (MyUri)queueURLS.Dequeue();
			}
			catch(Exception)
			{
			}
			Monitor.Exit(queueURLS);
			return uri;
		}

        public void RunParser()
		{
            ThreadsRunning = true;
            try
            {
                string strUri = ParseURL;//this.comboBoxWeb.Text.Trim();
                if (Directory.Exists(strUri) == true)
                    ParseFolder(strUri, 0);
                else
                {
                    if (File.Exists(strUri) == false)
                    {
                        Normalize(ref strUri);
                        //this.comboBoxWeb.Text = strUri;
                    }
                    MyUri uri = new MyUri(strUri);
                    this.EnqueueUri(uri, false);
                }
            }
            catch (Exception e)
            {
                LogError("Exception", e.Message);
                return;
            }
            
            //this.toolBarButtonContinue.Enabled = false;
            //this.buttonGo.Enabled = true;
		}
		
		private void Normalize(ref string strURL)
		{
			if(strURL.StartsWith("http://") == false)
				strURL = "http://"+strURL;
			if(strURL.IndexOf("/", 8) == -1)
				strURL += '/';
		}

		bool AddURL(ref MyUri uri)
		{
			foreach(string str in ExcludeHosts)
				if(str.Trim().Length > 0 && uri.Host.ToLower().IndexOf(str.Trim()) != -1)
				{
					LogError(uri.AbsoluteUri, "\r\nHost excluded as it includes reserved pattern ("+str+")");
					return false;
				}
			Monitor.Enter(urlStorage);
			bool bNew = false;
			try
			{
				string strURL = uri.AbsoluteUri;
				bNew = urlStorage.Add(ref strURL).Count == 1;
			}
			catch(Exception)
			{
			}
			Monitor.Exit(urlStorage);
			
			return bNew;
		}

        //void LogCell(ref ListViewItem itemLog, int nCell, string str)
        //{
        //    Monitor.Enter(this.listViewThreads);
        //    try
        //    {
        //        itemLog.SubItems[nCell].Text = str;
        //    }
        //    catch(Exception)
        //    {
        //    }
        //    Monitor.Exit(this.listViewThreads);
        //}
        private string StrStatus;
        public string strStatus
        {
            get 
            { 
                return this.StrStatus; 
            }
            set 
            {
                this.StrStatus = value;
                OnStatusChanged(this, value);
            }
        }
        public void ParseUri(MyUri uri, ref MyWebRequest request)
		{
            //string strStatus = "";
            //// check if connection is kept alive from previous connections or not
            if(request != null && request.response.KeepAlive)
                strStatus += "Connection live to: "+uri.Host+"\r\n\r\n";
            else
                strStatus += "Connecting: "+uri.Host+"\r\n\r\n";

            //ListViewItem itemLog = null;
            //Monitor.Enter(this.listViewThreads);
            //try
            //{	// update thread information in the threads view list
            //    itemLog = this.listViewThreads.Items[int.Parse(Thread.CurrentThread.Name)];
            //    int nDepth = uri.Depth;
            //    itemLog.SubItems[1].Text = nDepth.ToString();
            //    itemLog.ImageIndex = 1;
            //    itemLog.BackColor = Color.WhiteSmoke;
            //    // initialize status to Connect
            //    itemLog.SubItems[2].Text = "Connect";
            //    itemLog.ForeColor = Color.Red;
            //    itemLog.SubItems[3].Text = uri.AbsoluteUri;
            //    itemLog.SubItems[4].Text = "";
            //    itemLog.SubItems[5].Text = "";
            //}
            //catch (Exception)
            //{
            //}
            //Monitor.Exit(this.listViewThreads);

            try
            {
                // create web request
                request = MyWebRequest.Create(uri, request, KeepAlive);
                // set request timeout
                request.Timeout = RequestTimeout * 1000;
                // retrieve response from web request
                MyWebResponse response = request.GetResponse();
                // update status text with the request and response headers
                strStatus += request.Header + response.Header;

                // check for redirection
                if (response.ResponseUri.Equals(uri) == false)
                {
                    // add the new uri to the queue
                    this.EnqueueUri(new MyUri(response.ResponseUri.AbsoluteUri), true);
                    // update status
                    
                    strStatus += "Redirected to: " + response.ResponseUri + "\r\n";
                    // log current uri status
                    LogUri(uri.AbsoluteUri, strStatus);
                    // reset current request to avoid response socket opening case
                    request = null;
                    return;
                }

                // check for allowed MIME types
                if (AllMIMETypes == false && response.ContentType != null && MIMETypes.Length > 0)
                {
                    string strContentType = response.ContentType.ToLower();
                    int nExtIndex = strContentType.IndexOf(';');
                    if (nExtIndex != -1)
                        strContentType = strContentType.Substring(0, nExtIndex);
                    if (strContentType.IndexOf('*') == -1 && (nExtIndex = MIMETypes.IndexOf(strContentType)) == -1)
                    {
                        LogError(uri.AbsoluteUri, strStatus + "\r\nUnlisted Content-Type (" + strContentType + "), check settings.");
                        request = null;
                        return;
                    }
                    // find numbers
                    Match match = new Regex(@"\d+").Match(MIMETypes, nExtIndex);
                    int nMin = int.Parse(match.Value) * 1024;
                    match = match.NextMatch();
                    int nMax = int.Parse(match.Value) * 1024;
                    if (nMin < nMax && (response.ContentLength < nMin || response.ContentLength > nMax))
                    {
                        LogError(uri.AbsoluteUri, strStatus + "\r\nContentLength limit error (" + response.ContentLength + ")");
                        request = null;
                        return;
                    }
                }

                // check for response extention
                string[] ExtArray = { ".gif", ".jpg", ".css", ".zip", ".exe" };
                bool bParse = true;
                foreach (string ext in ExtArray)
                    if (uri.AbsoluteUri.ToLower().EndsWith(ext) == true)
                    {
                        bParse = false;
                        break;
                    }
                foreach (string ext in ExcludeFiles)
                    if (ext.Trim().Length > 0 && uri.AbsoluteUri.ToLower().EndsWith(ext) == true)
                    {
                        bParse = false;
                        break;
                    }

                // construct path in the hard disk
                string strLocalPath = uri.LocalPath;
                // check if the path ends with / to can crate the file on the HD 
                if (strLocalPath.EndsWith("/") == true)
                    // check if there is no query like (.asp?i=32&j=212)
                    if (uri.Query == "")
                        // add a default name for / ended pathes
                        strLocalPath += "default.html";
                // check if the uri includes a query string
                if (uri.Query != "")
                    // construct the name from the query hash value to be the same if we download it again
                    strLocalPath += uri.Query.GetHashCode() + ".html";
                // construct the full path folder
                string BasePath = this.Downloadfolder + "\\" + uri.Host + Path.GetDirectoryName(uri.AbsolutePath);
                // check if the folder not found
                if (Directory.Exists(BasePath) == false)
                    // create the folder
                    Directory.CreateDirectory(BasePath);
                // construct the full path name of the file
                string PathName = this.Downloadfolder + "\\" + uri.Host + strLocalPath.Replace("%20", " ");
                // open the output file
                FileStream streamOut = File.Open(PathName, FileMode.Create, FileAccess.Write, FileShare.ReadWrite);
                BinaryWriter writer = new BinaryWriter(streamOut);

                //itemLog.SubItems[2].Text = "Download";
                //itemLog.ForeColor = Color.Black;
                // receive response buffer
                string strResponse = "";
                byte[] RecvBuffer = new byte[10240];
                int nBytes, nTotalBytes = 0;
                // loop to receive response buffer
                while ((nBytes = response.socket.Receive(RecvBuffer, 0, 10240, SocketFlags.None)) > 0)
                {
                    // increment total received bytes
                    nTotalBytes += nBytes;
                    // write received buffer to file
                    writer.Write(RecvBuffer, 0, nBytes);
                    // check if the uri type not binary to can be parsed for refs
                    if (bParse == true)
                        // add received buffer to response string
                        strResponse += Encoding.ASCII.GetString(RecvBuffer, 0, nBytes);
                    // update view text
                    //itemLog.SubItems[4].Text = Commas(nTotalBytes);
                    //if (response.ContentLength > 0)
                        //itemLog.SubItems[5].Text = '%' + (100 - (response.ContentLength - nTotalBytes) * 100 / response.ContentLength).ToString();
                    // check if connection Keep-Alive to can break the loop if response completed
                    if (response.KeepAlive && nTotalBytes >= response.ContentLength && response.ContentLength > 0)
                        break;
                }
                // close output stream
                writer.Close();
                streamOut.Close();

                if (response.KeepAlive)
                    strStatus += "Connection kept alive to be used in subpages.\r\n";
                else
                {
                    // close response
                    response.Close();
                    strStatus += "Connection closed.\r\n";
                }
                // update status
                strStatus += Commas(nTotalBytes) + " bytes, downloaded to \"" + PathName + "\"\r\n";
                // increment total file count
                FileCount++;
                // increment total bytes count
                ByteCount += nTotalBytes;

                if (ThreadsRunning == true && bParse == true && uri.Depth < WebDepth)
                {
                    strStatus += "\r\nParsing page ...\r\n";

                    // check for restricted words
                    foreach (string strExcludeWord in ExcludeWords)
                        if (strExcludeWord.Trim().Length > 0 && strResponse.IndexOf(strExcludeWord) != -1)
                        {
                            LogError(uri.AbsoluteUri, strStatus + "\r\nPage includes reserved word (" + strExcludeWord + ")");
                            //EraseItem(itemLog);
                            File.Delete(PathName);
                            return;
                        }

                    // parse the page to search for refs
                    string strRef = @"(href|HREF|src|SRC)[ ]*=[ ]*[""'][^""'#>]+[""']";
                    MatchCollection matches = new Regex(strRef).Matches(strResponse);
                    strStatus += "Found: " + matches.Count + " ref(s)\r\n";
                    URLCount += matches.Count;
                    foreach (Match match in matches)
                    {
                        strRef = match.Value.Substring(match.Value.IndexOf('=') + 1).Trim('"', '\'', '#', ' ', '>');
                        try
                        {
                            if (strRef.IndexOf("..") != -1 || strRef.StartsWith("/") == true || strRef.StartsWith("http://") == false)
                                strRef = new Uri(uri, strRef).AbsoluteUri;
                            Normalize(ref strRef);
                            MyUri newUri = new MyUri(strRef);
                            if (newUri.Scheme != Uri.UriSchemeHttp && newUri.Scheme != Uri.UriSchemeHttps)
                                continue;
                            if (newUri.Host != uri.Host && KeepSameServer == true)
                                continue;
                            newUri.Depth = uri.Depth + 1;
                            if (this.EnqueueUri(newUri, true) == true)
                                strStatus += newUri.AbsoluteUri + "\r\n";
                        }
                        catch (Exception)
                        {
                        }
                    }
                }
                LogUri(uri.AbsoluteUri, strStatus);
            }
            catch (Exception e)
            {
                LogError(uri.AbsoluteUri, strStatus + e.Message);
                request = null;
            }
            //finally
            //{
            //    EraseItem(itemLog);
            //}
		}

        //void EraseItem(ListViewItem item)
        //{
        //    Monitor.Enter(this.listViewThreads);
        //    try
        //    {
        //        item.SubItems[1].Text = "";
        //        item.ImageIndex = 0;
        //        item.BackColor = Color.WhiteSmoke;
        //        item.ForeColor = Color.Black;
        //        item.SubItems[2].Text = "";
        //        item.SubItems[3].Text = "";
        //        item.SubItems[4].Text = "";
        //        item.SubItems[5].Text = "";
        //    }
        //    finally
        //    {
        //        Monitor.Exit(this.listViewThreads);
        //    }
        //}

		void LogUri(string strHead, string strBody)
		{
            //if(LastRequestCount > 0)
            //{
            //    Monitor.Enter(this.listViewRequests);
            //    try
            //    {
            //        ListViewItem item = this.listViewRequests.Items.Insert(0, DateTime.Now.ToString());
            //        item.SubItems.AddRange(new String[] { strHead, strBody });
            //        while(this.listViewRequests.Items.Count > LastRequestCount)
            //            this.listViewRequests.Items.RemoveAt(this.listViewRequests.Items.Count-1);
            //    }
            //    catch(Exception)
            //    {
            //    }
            //    Monitor.Exit(this.listViewRequests);
            //}
		}

		void LogError(string strHead, string strBody)
		{
            OnError(this, strHead, strBody);
            //Monitor.Enter(this.listViewErrors);
            //try
            //{
            //    ListViewItem item = this.listViewErrors.Items.Add((ErrorCount+1).ToString());
            //    item.SubItems.AddRange(new String[] { DateTime.Now.ToString(), strHead, strBody });
            //    while(this.listViewErrors.Items.Count > 1000)
            //        this.listViewErrors.Items.RemoveAt(0);
            //}
            //catch(Exception)
            //{
            //}
            //Monitor.Exit(this.listViewErrors);
            //ErrorCount++;
		}

		void ParseFolder(string folderName, int nDepth)
		{
			DirectoryInfo dir = new DirectoryInfo(folderName);
			FileInfo[] fia = dir.GetFiles("*.txt");
			foreach(FileInfo f in fia)
			{
				if(ThreadsRunning == false)
					break;
				MyUri uri = new MyUri(f.FullName);
				uri.Depth = nDepth;
				this.EnqueueUri(uri, true);
			}
			
			DirectoryInfo[] dia = dir.GetDirectories();
			foreach(DirectoryInfo d in dia)
			{
				if(ThreadsRunning == false)
					break;
				ParseFolder(d.FullName, nDepth+1);
			}
		}

        //private void toolBarMain_ButtonClick(object sender, System.Windows.Forms.ToolBarButtonClickEventArgs e)
        //{
        //    if(e.Button == this.toolBarButtonContinue)
        //        ContinueParsing();
        //    else	if(e.Button == this.toolBarButtonPause)
        //        PauseParsing();
        //    else	if(e.Button == this.toolBarButtonStop)
        //        StopParsing();
        //    else	if(e.Button == this.toolBarButtonDeleteAll)
        //        DeleteAllItems();
        //    else	if(e.Button == this.toolBarButtonSettings)
        //        ShowSettings(-1);
        //}
		
		void DeleteAllItems()
		{
            //if(MessageBox.Show(this, "Do you want to delete all?", "Verify", MessageBoxButtons.YesNo) == DialogResult.Yes)
            //{
            //    this.listViewErrors.Items.Clear();
            //    this.listViewRequests.Items.Clear();
            //    this.urlStorage = new SortTree();
            //    this.URLCount = 0;
            //    this.FileCount = 0;
            //    this.ByteCount = 0;
            //    this.ErrorCount = 0;
            //}
		}

		string Commas(int nNum)
		{
			string str = nNum.ToString();
			int nIndex = str.Length;
			while(nIndex > 3)
			{
				str = str.Insert(nIndex-3, ",");
				nIndex -= 3;
			}
			return str;
		}

        //private void CrawlerForm_Closing(object sender, System.ComponentModel.CancelEventArgs e)
        //{
        //    Settings.SetValue(this);
        //}

        //private void menuItemAbout_Click(object sender, System.EventArgs e)
        //{
        //    AboutForm form = new AboutForm();
        //    form.ShowDialog();
        //}

        //private void comboBoxWeb_KeyDown(object sender, System.Windows.Forms.KeyEventArgs e)
        //{
        //    if(e.KeyCode == Keys.Return)
        //    {
        //        if(threadParse == null || threadParse.ThreadState != ThreadState.Running)
        //            StartParsing();
        //    }
        //}

        //private void listViewErrors_SelectedIndexChanged(object sender, System.EventArgs e)
        //{
        //    if(this.listViewErrors.SelectedItems.Count == 0)
        //        return;
        //    ListViewItem item = this.listViewErrors.SelectedItems[0];
        //    this.textBoxErrorDescription.Text = item.SubItems[3].Text;
        //}

        //private void timerMem_Tick(object sender, System.EventArgs e)
        //{
        //    FreeMemory = ramCounter.NextValue();
        //    CPUUsage = (int)cpuCounter.NextValue();
        //}

        //private void menuItemFileMatches_Click(object sender, System.EventArgs e)
        //{
        //    ShowSettings(0);
        //}

        //private void menuItemOutput_Click(object sender, System.EventArgs e)
        //{
        //    ShowSettings(1);
        //}

        //private void menuItemConnections_Click(object sender, System.EventArgs e)
        //{
        //    ShowSettings(2);
        //}

        //private void menuItemAdvanced_Click(object sender, System.EventArgs e)
        //{
        //    ShowSettings(3);
        //}

        //private void menuItemSettingsFileMatches_Click(object sender, System.EventArgs e)
        //{
        //    ShowSettings(0);
        //}

        //private void menuItemSettingsOutput_Click(object sender, System.EventArgs e)
        //{
        //    ShowSettings(1);
        //}

        //private void menuItemSettingsConnections_Click(object sender, System.EventArgs e)
        //{
        //    ShowSettings(2);
        //}

        //private void menuItemSettingsAdvanced_Click(object sender, System.EventArgs e)
        //{
        //    ShowSettings(3);
        //}

        //private void menuItemCut_Click(object sender, System.EventArgs e)
        //{
        //    //Clipboard.SetDataObject(this.comboBoxWeb.SelectedText);
        //    //this.comboBoxWeb.SelectedText = "";
        //}

        //private void menuItemCopy_Click(object sender, System.EventArgs e)
        //{
        //    //Clipboard.SetDataObject(this.comboBoxWeb.SelectedText);
        //}

        //private void menuItemPaste_Click(object sender, System.EventArgs e)
        //{
        //    //IDataObject obj = Clipboard.GetDataObject();
        //    //if(obj != null)
        //    //    this.comboBoxWeb.SelectedText = obj.GetData("System.String").ToString();
        //}

        //private void menuItemDelete_Click(object sender, System.EventArgs e)
        //{
        //    //this.comboBoxWeb.SelectedText = "";
        //}

        //private void listViewRequests_SelectedIndexChanged(object sender, System.EventArgs e)
        //{
        //    //if(this.listViewRequests.SelectedItems.Count == 0)
        //    //    return;
        //    //ListViewItem item = this.listViewRequests.SelectedItems[0];
        //    //if(item.SubItems.Count > 2)
        //    //    this.textBoxRequest.Text = item.SubItems[2].Text;
        //}

        //private void menuItemHttp_Click(object sender, System.EventArgs e)
        //{
        //    //this.comboBoxWeb.Text = "http://";
        //}

        //private void menuItemFileBrowse_Click(object sender, System.EventArgs e)
        //{
        //    OnFileBrowse();
        //}
		
		void OnFileBrowse()
		{
            //OpenFileDialog dlg = new OpenFileDialog();
            //dlg.Title = "Select file to parse";
            //dlg.Filter = "Text files (*.txt)|*.txt|All files (*.*)|*.*";
            //if(dlg.ShowDialog() == DialogResult.OK)
            //    this.comboBoxWeb.Text = dlg.FileName;
		}

        //private void toolBarWeb_ButtonDropDown(object sender, System.Windows.Forms.ToolBarButtonClickEventArgs e)
        //{
        //    this.menuItemBrowseHttp = new System.Windows.Forms.MenuItem();
        //    e.Button.DropDownMenu = new System.Windows.Forms.ContextMenu();
        //    e.Button.DropDownMenu.MenuItems.AddRange(new System.Windows.Forms.MenuItem[] {
        //                                                                                      this.menuItemBrowseHttp});
        //    // 
        //    // menuItemBrowseHttp
        //    // 
        //    this.menuItemBrowseHttp.Index = 0;
        //    this.menuItemBrowseHttp.MenuItems.AddRange(new System.Windows.Forms.MenuItem[] {
        //                                                                                       this.menuItemHttp,
        //                                                                                       this.menuItem5});
        //    this.menuItemBrowseHttp.Text = "&Http";

        //    foreach (string item in this.comboBoxWeb.Items)
        //    {
        //        try
        //        {
        //            Uri uri = new Uri(item);
        //            MenuItem mi = new MenuItem(item, new System.EventHandler(OnBrowse));
        //            if(File.Exists(item) == true)
        //                e.Button.DropDownMenu.MenuItems[2].MenuItems.Add(mi);
        //            else	if(Directory.Exists(item) == true)
        //                e.Button.DropDownMenu.MenuItems[1].MenuItems.Add(mi);
        //            else	if(uri.Scheme == Uri.UriSchemeHttp)
        //                e.Button.DropDownMenu.MenuItems[0].MenuItems.Add(mi);
        //        }
        //        catch(Exception)
        //        {
        //        }
        //    }
        //}


		private void menuItemListClear_Click(object sender, System.EventArgs e)
		{
            //switch(this.tabControlRightView.SelectedTab.Text)
            //{
            //    case "Threads":
            //        this.listViewThreads.Items.Clear();
            //        break;
            //    case "Requests":
            //        this.listViewRequests.Items.Clear();
            //        break;
            //    case "Errors":
            //        this.listViewErrors.Items.Clear();
            //        break;
            //}
		}

		private void timerConnectionInfo_Tick(object sender, System.EventArgs e)
		{
			ConnectionInfo();
		}

	}

	public class MyUri : System.Uri
	{
		public MyUri(string uriString):base(uriString)
		{
		}
		public int Depth;
	}

	public class MyWebRequest
	{
		public MyWebRequest(Uri uri, bool bKeepAlive)
		{
			Headers = new WebHeaderCollection();
			RequestUri = uri;
			Headers["Host"] = uri.Host;
			KeepAlive = bKeepAlive;
			if(KeepAlive)
				Headers["Connection"] = "Keep-Alive";
			Method = "GET";
		}
		public static MyWebRequest Create(Uri uri, MyWebRequest AliveRequest, bool bKeepAlive)
		{
			if( bKeepAlive &&
				AliveRequest != null &&
				AliveRequest.response != null &&
				AliveRequest.response.KeepAlive && 
				AliveRequest.response.socket.Connected && 
				AliveRequest.RequestUri.Host == uri.Host)
			{
				AliveRequest.RequestUri = uri;
				return AliveRequest;
			}
			return new MyWebRequest(uri, bKeepAlive);
		}
		public MyWebResponse GetResponse()
		{
			if(response == null || response.socket == null || response.socket.Connected == false)
			{
				response = new MyWebResponse();
				response.Connect(this);
				response.SetTimeout(Timeout);
			}
			response.SendRequest(this);
			response.ReceiveHeader();
			return response;
		}

		public int Timeout;
		public WebHeaderCollection Headers;
		public string Header;
		public Uri RequestUri;
		public string Method;
		public MyWebResponse response;
		public bool KeepAlive;
	}
	public class MyWebResponse
	{
		public MyWebResponse()
		{
		}
		public void Connect(MyWebRequest request)
		{
			ResponseUri = request.RequestUri;
			
			socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
			IPEndPoint remoteEP = new IPEndPoint(Dns.Resolve(ResponseUri.Host).AddressList[0], ResponseUri.Port);
			socket.Connect(remoteEP);			
		}
		public void SendRequest(MyWebRequest request)
		{
			ResponseUri = request.RequestUri;

			request.Header = request.Method+" "+ResponseUri.PathAndQuery+" HTTP/1.0\r\n"+request.Headers;
			socket.Send(Encoding.ASCII.GetBytes(request.Header));
		}
		public void SetTimeout(int Timeout)
		{
			socket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.SendTimeout, Timeout*1000);
			socket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReceiveTimeout, Timeout*1000);
		}
		public void ReceiveHeader()
		{
			Header = "";
			Headers = new WebHeaderCollection();

			byte[] bytes = new byte[10];
			while(socket.Receive(bytes, 0, 1, SocketFlags.None) > 0)
			{
				Header += Encoding.ASCII.GetString(bytes, 0, 1);
				if(bytes[0] == '\n' && Header.EndsWith("\r\n\r\n"))
					break;
			}
			MatchCollection matches = new Regex("[^\r\n]+").Matches(Header.TrimEnd('\r', '\n'));
			for(int n = 1; n < matches.Count; n++)
			{
				string[] strItem = matches[n].Value.Split(new char[] { ':' }, 2);
				if(strItem.Length > 0)
					Headers[strItem[0].Trim()] = strItem[1].Trim();
			}
			// check if the page should be transfered to another location
			if( matches.Count > 0 && (
				matches[0].Value.IndexOf(" 302 ") != -1 || 
				matches[0].Value.IndexOf(" 301 ") != -1))
				// check if the new location is sent in the "location" header
				if(Headers["Location"] != null)
				{
					try		{	ResponseUri = new Uri(Headers["Location"]);		}
					catch	{	ResponseUri = new Uri(ResponseUri, Headers["Location"]);		}
				}
			ContentType = Headers["Content-Type"];
			if(Headers["Content-Length"] != null)
				ContentLength = int.Parse(Headers["Content-Length"]);
			KeepAlive = (Headers["Connection"] != null && Headers["Connection"].ToLower() == "keep-alive") ||
						(Headers["Proxy-Connection"] != null && Headers["Proxy-Connection"].ToLower() == "keep-alive");
		}
		public void Close()
		{
			socket.Close();
		}
		public Uri ResponseUri;
		public string ContentType;
		public int ContentLength;
		public WebHeaderCollection Headers; 
		public string Header;
		public Socket socket;
		public bool KeepAlive;
	}
}
